package com.google.code.sapwcrawler.datahandler;

import com.google.code.sapwcrawler.data.ParsedHtml;
import com.google.code.sapwcrawler.data.html.Link;
import com.google.code.sapwcrawler.download.cycle.DownloadCycle;
import com.google.code.sapwcrawler.urlfilter.URLFilter;

import java.net.*;

public class URLFinder implements DataHandler<ParsedHtml>
{
	private DownloadCycle cycle;
	private URLFilter urlFilter;
	public void setDownloadCycle(DownloadCycle v)
	{
		this.cycle = v;
	}
	
	public void setURLFilter(URLFilter v)
	{
		this.urlFilter = v;
	}
	
	public void processData(ParsedHtml data)
	{
		for(Link link: data.getLinks())	
			add(link);
	}
	
	public void add(Link link)
	{
		if(!(link.getRel() != null && link.getRel().equals("nofollow")))
			add(link.getHref());
	}
	
	public void add(URL url)
	{
		if(urlFilter.isDownloadableURL(url))
			cycle.addUrl(url);
	}
}
